﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Data;
using MathNet.Numerics.Statistics;
using System.IO;
using System.Collections.Specialized;
using System.Text.RegularExpressions;
using System.Diagnostics;

namespace Cadmon
{
    enum Period { Day = 86400, Week = 604800 };

    class Program
    {
        DataTable m_data;

        // Parameters
        int m_TimeWindow = 1800; //sec
        Period m_period = Period.Day;
        double m_dSDevMultiplier = 3;
        double m_dSDevMinValue = 0.07;
        double m_dAbruptChangeValueThreshold = 1; // GB/s
        int m_nMinDataIntervalLength = 216; // points that is enough for takijng statistics
        int m_nSubsequentAlerts = 4;
        bool m_bDiscardTodayData = false;

        public string m_strOutFileName = "";

        static void Main(string[] args)
        {
            if (args.Count() == 0)
            {
                Console.WriteLine("Usage: rrdtool fetch {someFile}.rrd [AVERAGE|LAST|...] -s -7d | [mono] cadmon.exe -c configFileName [-o resultFileNameWithoutExtension] [-d [DateTime]] ");
                return;
            }

            Arguments commandLine = new Arguments(args);

            if (commandLine["c"] == null)
            {
                Console.WriteLine("Configuration file not specified. Use cadmon.exe -c configFileName");
                return;
            }

            Program prg = new Program(commandLine["c"]);

            if (commandLine["o"] != null)
            {
                prg.m_strOutFileName = commandLine["o"];
                prg.Check();
                return;
            }

            if (commandLine["d"] != null)
            {
                if (commandLine["d"] == "true")
                {
                    prg.DumpCorrespondencePoints(DateTime.UtcNow);
                }
                else
                {
                    DateTime dt;
                    if (DateTime.TryParse(commandLine["d"], out dt))
                        prg.DumpCorrespondencePoints(dt);
                }

                return;
            }

            Console.WriteLine("Usage: rrdtool fetch {someFile}.rrd [AVERAGE|LAST|...] -s -7d | [mono] cadmon.exe -c configFileName [-o resultFileNameWithoutExtension] [-d [DateTime]] ");
        }
        public Program(string strConfigFileName)
        {
            LoadConfiguration(strConfigFileName);
            DumpParameters();
        }
        public void Check()
        {
            //Console.WriteLine("Sigma: {0}", this.m_dSDevMultiplier);

            // Get data from STDIN, parse and put it into m_arrData array.
            //m_arrData = GetDataFromSTDIN();
            m_data = GetDataFromSTDIN();

            int nNumberOfDataColumns = GetDataColumnsIDCollection().Count();

            DataRow lastDataRow = m_data.NewRow();
            lastDataRow.ItemArray = (object[])m_data.Rows[m_data.Rows.Count - 1].ItemArray.Clone();// CLone the newest row

            Console.WriteLine("Analysing last RRD data based on it's history.\r\n");
            Console.WriteLine("DateTime: {0}", lastDataRow["Datetime"]);
            Console.WriteLine("Data (Gbit/s): {0}", ConcatenateData(lastDataRow));
            Console.WriteLine();
            Console.WriteLine("Number of data columns to analyze: {0}", nNumberOfDataColumns);

            // Calculate time interval that included datetime.now, i.e. CurTime in [t0,t0+m_TimeWindow]. 
            // Get data points from all days in history that are included in [t0,t0+m_TimeWindow] interval and put in one array.
            //Console.WriteLine("Entering RemoveNonCorrespondencePoints()");

            RemoveNonCorrespondencePoints(null);

            List<List<double>> arrPoints = GetLastQuasiStationarInterval();

            #region Calculate statistics for each datacolumn
            Console.WriteLine();
            // calculate mean and variance. They need to be multiplied by 8
            //List<DescriptiveStatistics> arrStatistics = new List<DescriptiveStatistics>();
            List<double> arrMean = new List<double>();
            List<double> arrSDev = new List<double>();

            int k = 0;
            foreach (var arr in arrPoints)
            {
                arrMean.Add(arr.Mean());

                // Implement stdev min value
                var sdev = arr.StandardDeviation();
                if (sdev > m_dSDevMinValue)
                    arrSDev.Add(sdev);
                else
                    arrSDev.Add(m_dSDevMinValue);

                Console.WriteLine("Mean[Data{2}] (Gbit/s): {0}\r\nStDev[Data{2}] (Gbit/s): {1}", arrMean[k], arrSDev[k], k);
                k++;
            }

            #endregion

            #region Check if Last Point is in mean+-M*sigma
            Console.WriteLine();

            // Check for 4 consecutive alerts.
            //List<int> arrCounter = new List<int>(nNumberOfDataColumns);
            double dLowerInterval, dUpperInterval;

            for (int i = 0; i < nNumberOfDataColumns; i++)
            {
                dLowerInterval = arrMean[i] - m_dSDevMultiplier * arrSDev[i];
                dUpperInterval = arrMean[i] + m_dSDevMultiplier * arrSDev[i];

                Console.WriteLine("Parameter {0} normal interval is {1} - {2}, delta = {3}", i, dLowerInterval, dUpperInterval, dUpperInterval - dLowerInterval);
                if ((double)lastDataRow["Data" + i] > arrMean[i] + m_dSDevMultiplier * arrSDev[i])
                {
                    Console.WriteLine("-------------------------> ALERT: value {0} is anomalous high! <-------------------------", (double)lastDataRow["Data" + i]);
                    SaveToFile("1", m_strOutFileName, i);
                }
                else
                    if ((double)lastDataRow["Data" + i] < arrMean[i] - m_dSDevMultiplier * arrSDev[i])
                    {
                        Console.WriteLine("-------------------------> ALERT: value {0} is anomalous low! <-------------------------", (double)lastDataRow["Data" + i]);
                        SaveToFile("1", m_strOutFileName, i);
                    }
                    else
                    {
                        Console.WriteLine("Value {0} is within it's normal interval.", (double)lastDataRow["Data" + i]);
                        SaveToFile("0", m_strOutFileName, i);
                    }
            }

            #endregion

            Console.WriteLine("-----------------------------------");
        }
        public void DumpCorrespondencePoints(DateTime dt)
        {
            m_data = GetDataFromSTDIN();
            RemoveNonCorrespondencePoints(dt);

            foreach (string dataColumnName in GetDataColumnsIDCollection())
            {
                var arrNumbersCollection = from DataRow row in m_data.Rows
                                           select (double)row[dataColumnName];
                double[] arrNumbers = arrNumbersCollection.ToArray();

                DumpDataToFile(arrNumbersCollection.ToArray(), dataColumnName + ".txt");
            }
            //for (int i = 0; i < arrPoints.Count; i++)
            //    DumpDataToFile(arrPoints[i], "Dump_" + i + ".txt");
        }

        void LoadConfiguration(string strConfigFileName)
        {
            string[] lines;

            try
            {
                //Console.WriteLine(strConfigFileName);
                Console.WriteLine("Config file name: " + strConfigFileName);

                if (File.Exists(strConfigFileName))
                {
                    lines = System.IO.File.ReadAllLines(strConfigFileName);
                }
                else
                {
                    Console.WriteLine("Error: configuration file nto found, will use default values.");
                    lines = new string[0];
                }

                foreach (string line in lines)
                {
                    // Remove comments
                    int nCommentIndex = line.IndexOf('#');
                    string[] keyValuePair;

                    if (nCommentIndex > 0)
                        keyValuePair = line.Substring(0, nCommentIndex).Split('=');
                    else
                        keyValuePair = line.Split('=');

                    if (keyValuePair.Count() == 2)
                    {
                        //Console.WriteLine(keyValuePair[0].Trim().ToLower());
                        switch (keyValuePair[0].Trim().ToLower())
                        {
                            case "timewindow":
                                try
                                {
                                    m_TimeWindow = Convert.ToInt32(keyValuePair[1].Trim());
                                }
                                catch (Exception e) { Console.WriteLine(e); }
                                break;
                            case "period":
                                if (keyValuePair[1].Trim().ToLower() == "day")
                                    m_period = Period.Day;

                                if (keyValuePair[1].Trim().ToLower() == "week")
                                    m_period = Period.Week;
                                break;
                            case "sdevmultiplier":
                                try
                                {
                                    m_dSDevMultiplier = Convert.ToDouble(keyValuePair[1].Trim());
                                }
                                catch { }
                                break;
                            case "sdevminvalue":
                                try
                                {
                                    m_dSDevMinValue = Convert.ToDouble(keyValuePair[1].Trim());
                                }
                                catch { }
                                break;
                            case "subsequentalerts":
                                try
                                {
                                    m_nSubsequentAlerts = Convert.ToInt32(keyValuePair[1].Trim());
                                }
                                catch { }
                                break;
                            case "discardtodaydata":
                                try
                                {
                                    m_bDiscardTodayData = Convert.ToBoolean(keyValuePair[1].Trim());
                                }
                                catch { };
                                break;
                            case "abruptchangevaluethreshold":
                                try
                                {
                                    m_dAbruptChangeValueThreshold = Convert.ToDouble(keyValuePair[1].Trim());
                                }
                                catch (Exception e) { Console.WriteLine(e); }

                                break;
                            case "mindataintervallength":
                                try
                                {
                                    m_nMinDataIntervalLength = Convert.ToInt32(keyValuePair[1].Trim());
                                }
                                catch { }
                                break;

                        }
                    }
                }
                //Console.WriteLine("Sigma = {0}", m_dSDevMultiplier);
            }
            catch
            {
                throw;
            }
        }
        DataTable GetDataFromSTDIN()
        {
            DataTable dt = new DataTable();
            dt.Columns.Add("Datetime", System.Type.GetType("System.DateTime"));
            dt.Columns.Add("Mark", System.Type.GetType("System.Boolean")); // Add Mark column

            string s;
            DateTime datetime;
            List<double> arrData;
            int i = 0;

            while ((s = Console.ReadLine()) != null)
            {
                if (s == "exit")
                    break;

                if (!ParseInputData(s, out datetime, out arrData))
                    continue;

                // Initialize datatable according to the obtained data
                if (i == 0)
                    for (int j = 0; j < arrData.Count(); j++)
                        dt.Columns.Add("Data" + j, System.Type.GetType("System.Double"));

                DataRow row = dt.NewRow();
                row["Datetime"] = datetime;
                for (int j = 0; j < arrData.Count(); j++)
                {
                    row["Data" + j] = arrData[j];
                }
                dt.Rows.Add(row);

                i++;
            }

            return dt;
        }
        void RemoveNonCorrespondencePoints(DateTime? dt)
        {
            //int nNumberOfDataColumns = m_data.Columns.Count;
            DataRow lastDataRow = m_data.Rows[m_data.Rows.Count - 1]; // The newest point in data
            DataRow firstDataRow = m_data.Rows[0]; // The oldes point in data

            //  Calculate start point t0 of [t0,t0+m_TimeWindow] interval, which include current datetime
            DateTime startDT;
            if (dt != null)
                startDT = GetNeigbourTimeSpans((DateTime)dt);
            else
                startDT = GetNeigbourTimeSpans((DateTime)lastDataRow["Datetime"]);

            // Get first record's TimeStamp (the limit) of RRD database
            DateTime firsRecordTS = GetStartOfDay((DateTime)firstDataRow["Datetime"]);

            // Do not consider points for the current day
            startDT = startDT.AddSeconds(-(int)m_period);

            // Iterate thru all available intervals
            while (startDT >= firsRecordTS)
            {
                if (IsWorkingDay((DateTime)lastDataRow["Datetime"]) == IsWorkingDay(startDT))
                {
                    // Add data from the current interval to the storage
                    //Console.WriteLine("Entering MarkNonCorrespondencePoints()");
                    MarkNonCorrespondencePoints(startDT);
                }

                // Reduce period (in seconds) from time left TS neigbour
                startDT = startDT.AddSeconds(-(int)m_period);
            }

            //Console.WriteLine("Entering RemoveNonMarkedPoints()");

            RemoveNonMarkedPoints();
        }

        List<List<double>> GetLastQuasiStationarInterval()
        {
            List<List<double>> output = new List<List<double>>();

            //List<string> dataIndexCollection = GetDataColumnsIDCollection();
            foreach (string dataColumnName in GetDataColumnsIDCollection())
            {
                // Initialize diff array 
                List<int> changePoints = new List<int>();

                // Add first point coordinate
                changePoints.Add(0);

                // Get all points of current column
                var arrNumbersCollection = from DataRow row in m_data.Rows
                                           select (double)row[dataColumnName];
                double[] arrNumbers = arrNumbersCollection.ToArray();


                for (int j = 1; j < arrNumbers.Count(); j++)
                {
                    // Differentiate and consider absolute value.
                    double dDiffValue = Math.Abs(arrNumbers[j] - arrNumbers[j - 1]);

                    // Store coordinates of extremum points.
                    if (dDiffValue >= m_dAbruptChangeValueThreshold)
                        changePoints.Add(j);
                }

                // Add first point coordinate
                changePoints.Add(arrNumbers.Count() - 1);

                Console.WriteLine();
                Console.WriteLine("{0} abrupt change points found in traffic data {1} pattern", changePoints.Count() - 2, dataColumnName);

                // Determine the most closest to now interval
                int[] dArrChangePoints = changePoints.ToArray();
                int d1 = 0, d2 = 0;

                for (int j = dArrChangePoints.Count() - 1; j > 0; j--)
                {
                    if (dArrChangePoints[j] - dArrChangePoints[j - 1] >= m_nMinDataIntervalLength)
                    {
                        d1 = dArrChangePoints[j - 1];
                        d2 = dArrChangePoints[j];
                        break;
                    }
                }

                // If there is no quasi stable interval, then forced shoudl use all interval
                if (d1 == 0 && d2 == 0)
                {
                    d2 = arrNumbers.Count() - 1;
                }

                // Add points corresponding to the most closest interval
                List<double> lst = new List<double>();
                for (int j = d1; j <= d2; j++)
                {
                    lst.Add(arrNumbers[j]);
                }

                // in m_arrData the order of points is reversed compared to input
                Console.WriteLine("Normal traffic befaviour will be learned from {0} - {1} interval. Number of points {2}", m_data.Rows[d1]["Datetime"], m_data.Rows[d2]["Datetime"], lst.Count());
                output.Add(lst);
            }

            return output;
        }
        DateTime GetNeigbourTimeSpans(DateTime dt)
        {
            DateTime dblStartInterval = GetStartOfDay(dt);
            DateTime dblStopInterval = GetEndOfDay(dt);

            for (DateTime interval = dblStartInterval; interval < dblStopInterval; interval = interval.AddSeconds(m_TimeWindow))
            {
                if (dt >= interval && dt < interval.AddSeconds(m_TimeWindow))
                    return interval;
            }

            return NullDate();
        }
        void MarkNonCorrespondencePoints(DateTime dt)
        {
            int i = 0;
            foreach (DataRow row in m_data.Rows)
                if ((DateTime)row["Datetime"] >= dt && (DateTime)row["Datetime"] < dt.AddSeconds(m_TimeWindow))
                {
                    row.BeginEdit();
                    row["Mark"] = true;
                    i++;
                }

            Console.WriteLine("Points found in correspondence interval [{0}, {1}): {2}", dt, dt.AddSeconds(m_TimeWindow), i);
        }
        void RemoveNonMarkedPoints()
        {
            DataTable dt = m_data.Clone();

            foreach (DataRow dRow in m_data.Rows)
                if (dRow["Mark"] != DBNull.Value && (bool)dRow["Mark"] == true)
                {
                    var newRow = dt.NewRow();
                    newRow.ItemArray = (object[])dRow.ItemArray.Clone();
                    dt.Rows.Add(newRow);
                }

            m_data = dt;

            //List<DataRow> rowsToDelete = new List<DataRow>();

            //foreach (DataRow dRow in m_data.Rows)
            //    if (dRow["Mark"] == DBNull.Value || (bool)dRow["Mark"] == false)
            //        rowsToDelete.Add(dRow);

            //foreach (DataRow dRow in rowsToDelete)
            //    m_data.Rows.Remove(dRow);
        }

        // Helper methods
        DateTime GetStartOfDay(DateTime dt)
        {
            return dt.Date;
        }
        DateTime GetEndOfDay(DateTime dt)
        {
            return dt.Date.AddDays(1);
        }
        DateTime NullDate()
        {
            return new DateTime(1970, 1, 1, 0, 0, 0);
        }
        void SaveToFile(string strRes, string strFileName, int nParamID)
        {
            try
            {
                if (strFileName == "")
                    return;

                strFileName += "." + nParamID.ToString() + ".txt";

                List<string> arrTextToSave = new List<string>();

                string[] lines;
                if (File.Exists(strFileName))
                    lines = System.IO.File.ReadAllLines(strFileName);
                else
                    lines = new string[0];

                if (lines.Count() < m_nSubsequentAlerts)
                {
                    arrTextToSave.AddRange(lines);
                }
                else
                {
                    for (int i = lines.Count() - m_nSubsequentAlerts + 1; i < lines.Count(); i++)
                    {
                        arrTextToSave.Add(lines[i]);
                    }
                }
                arrTextToSave.Add(strRes);

                System.IO.File.WriteAllLines(strFileName, arrTextToSave);
            }
            catch
            { }

        }

        void DumpDataToFile(double[] input, string filename)
        {
            TextWriter tw = new StreamWriter(filename);

            for (int i = 0; i < input.Count(); i++)
            {
                tw.WriteLine(input[i]);
            }

            tw.Close();
        }
        void DumpParameters()
        {
            Console.WriteLine("m_TimeWindow {0}", m_TimeWindow);
            Console.WriteLine("m_period {0}", m_period);
            Console.WriteLine("m_dSDevMultiplier {0}", m_dSDevMultiplier);
            Console.WriteLine("m_dSDevMinValue {0}", m_dSDevMinValue);
            Console.WriteLine("m_dAbruptChangeValueThreshold {0}", m_dAbruptChangeValueThreshold);
            Console.WriteLine("m_nMinDataIntervalLength {0}", m_nMinDataIntervalLength);
            Console.WriteLine("m_nSubsequentAlerts {0}", m_nSubsequentAlerts);
            Console.WriteLine("m_bDiscardTodayData {0}", m_bDiscardTodayData);
        }
        /// <summary>
        /// Convert Unix Timestamp to datetime
        /// </summary>
        /// <param name="timespan"></param>
        /// <returns>DateTime</returns>
        DateTime UTS2DateTime(long timespan)
        {
            return (new DateTime(1970, 1, 1, 0, 0, 0)).AddSeconds(timespan);
        }
        bool IsWorkingDay(DateTime dt)
        {
            if (dt.DayOfWeek == DayOfWeek.Monday || dt.DayOfWeek == DayOfWeek.Tuesday || dt.DayOfWeek == DayOfWeek.Wednesday || dt.DayOfWeek == DayOfWeek.Thursday || dt.DayOfWeek == DayOfWeek.Friday)
                return true;
            else
                return false;
        }

        // DataTable methods
        bool ParseInputData(string strRow, out DateTime datetime, out List<double> arrData)
        {
            arrData = new List<double>();
            datetime = new DateTime(1970, 1, 1, 0, 0, 0);

            try
            {
                string[] arrRow = strRow.Split(' ');
                datetime = UTS2DateTime(Convert.ToInt64(arrRow[0].TrimEnd(':')));
                //unixTimespan = Convert.ToInt64(arrRow[0].TrimEnd(':'));

                for (int i = 1; i < arrRow.Length; i++)
                {
                    arrData.Add(Double.Parse(arrRow[i], System.Globalization.NumberStyles.Float) / 134217728);
                }

                return true;
            }
            catch
            { }

            return false;
        }
        string ConcatenateData(DataRow dr)
        {
            string output = "";

            foreach (string dataColumnName in GetDataColumnsIDCollection())
            {
                output += dr[dataColumnName].ToString() + ", ";
            }

            return output.Substring(0, output.Length - 2);
        }
        List<string> GetDataColumnsIDCollection()
        {
            List<string> output = new List<string>();

            foreach (DataColumn dc in m_data.Columns)
                if (dc.ColumnName.Contains("Data"))
                    output.Add(dc.ColumnName);

            return output;
        }
    }
    class SDataRow
    {
        double m_UnixTimespan = 0;
        List<double> m_arrData;

        public SDataRow(string row)
        {
            try
            {
                m_arrData = new List<double>();

                string[] arrRow = row.Split(' ');
                m_UnixTimespan = Convert.ToInt64(arrRow[0].TrimEnd(':'));

                for (int i = 1; i < arrRow.Length; i++)
                {
                    m_arrData.Add(Double.Parse(arrRow[i], System.Globalization.NumberStyles.Float) / 134217728);
                }
            }
            catch
            { }
        }
        public string DumpData()
        {
            string ret = "";
            foreach (double val in m_arrData)
            {
                ret += val.ToString();
                ret += ", ";
            }

            return ret.Substring(0, ret.Length - 2);
        }

        public double Timespan
        {
            get
            { return m_UnixTimespan; }
        }
        public List<double> Data
        {
            get
            { return m_arrData; }
        }
    }
    public class Arguments
    {
        // Variables
        private StringDictionary Parameters;

        // Constructor
        public Arguments(string[] Args)
        {
            Parameters = new StringDictionary();
            //Regex Spliter = new Regex(@"^-{1,2}|^/|=|:",
            Regex Spliter = new Regex(@"^-{1,2}|=",
                RegexOptions.IgnoreCase | RegexOptions.Compiled);

            Regex Remover = new Regex(@"^['""]?(.*?)['""]?$",
                RegexOptions.IgnoreCase | RegexOptions.Compiled);

            string Parameter = null;
            string[] Parts;

            // Valid parameters forms:
            // {-,/,--}param{ ,=,:}((",')value(",'))
            // Examples: 
            // -param1 value1 --param2 /param3:"Test-:-work" 
            //   /param4=happy -param5 '--=nice=--'
            foreach (string Txt in Args)
            {
                // Look for new parameters (-,/ or --) and a
                // possible enclosed value (=,:)
                Parts = Spliter.Split(Txt, 3);

                switch (Parts.Length)
                {
                    // Found a value (for the last parameter 
                    // found (space separator))
                    case 1:
                        if (Parameter != null)
                        {
                            if (!Parameters.ContainsKey(Parameter))
                            {
                                Parts[0] =
                                    Remover.Replace(Parts[0], "$1");

                                Parameters.Add(Parameter, Parts[0]);
                            }
                            Parameter = null;
                        }
                        // else Error: no parameter waiting for a value (skipped)
                        break;

                    // Found just a parameter
                    case 2:
                        // The last parameter is still waiting. 
                        // With no value, set it to true.
                        if (Parameter != null)
                        {
                            if (!Parameters.ContainsKey(Parameter))
                                Parameters.Add(Parameter, "true");
                        }
                        Parameter = Parts[1];
                        break;

                    // Parameter with enclosed value
                    case 3:
                        // The last parameter is still waiting. 
                        // With no value, set it to true.
                        if (Parameter != null)
                        {
                            if (!Parameters.ContainsKey(Parameter))
                                Parameters.Add(Parameter, "true");
                        }

                        Parameter = Parts[1];

                        // Remove possible enclosing characters (",')
                        if (!Parameters.ContainsKey(Parameter))
                        {
                            Parts[2] = Remover.Replace(Parts[2], "$1");
                            Parameters.Add(Parameter, Parts[2]);
                        }

                        Parameter = null;
                        break;
                }
            }
            // In case a parameter is still waiting
            if (Parameter != null)
            {
                if (!Parameters.ContainsKey(Parameter))
                    Parameters.Add(Parameter, "true");
            }
        }

        // Retrieve a parameter value if it exists 
        // (overriding C# indexer property)
        public string this[string Param]
        {
            get
            {
                return (Parameters[Param]);
            }
        }
    }
}
